#coding=utf-8
import cmath,urllib.request,os,math,random,sys,time,threading,json
import queue
import datetime
import requests
from io import BytesIO
import base64
import sqlite3

class Cdt:
    def __str__(self):
        return '{x = %s,y = %s}'%(self.x,self.y)        
    
def getPixelFromCdt(x,y,z):
    #'''根据经纬度坐标以及缩放等级获取像素坐标'''
    pixel = Cdt()
    sinLatitude = cmath.sin(y * cmath.pi / 180)
    pixel.x = ((x + 180) / 360) * 256 * (2**z)
    temp = cmath.log((1+sinLatitude)/(1-sinLatitude))
    pixel.y = abs((0.5 - temp / (4 * cmath.pi))*256*(2**z))
    return pixel

def getTileFromPixel(pixel):
    #'''根据像素坐标获取切片'''
    tile = Cdt()
    tile.x = math.floor(pixel.x / 256)
    tile.y = math.floor(pixel.y / 256)
    return tile

   
def createCacheStruc(extent,lvRange,cacheDir,out_type):
    #'''创建缓存目录结构及计算tile'''

    print (u'创建Cache目录及计算tile数目...')

    for lv in lvRange:

        startTile = getTileFromPixel(getPixelFromCdt(extent[0],extent[1],lv))
        endTile = getTileFromPixel(getPixelFromCdt(extent[2],extent[3],lv))
        #print(startTile)
        xRange = range(int(startTile.x),int(endTile.x))
        yRange = range(int(startTile.y),int(endTile.y))
        for row in yRange:
            rowName = cacheDir +os.sep + str(lv) + os.sep + str(row)
            if not os.path.exists(rowName) and out_type == 'file':
                os.makedirs(rowName)  
            
            for col in xRange:
                tempTask.put('%s,%s,%s,%s,%d'%(lv,row,col,out_type,0))


def createRemoteUrl(x,y,z):
    #'''创建远程tile地址'''
    port = str(random.randint(1,4))
    x = str(x)
    y = str(y)
    z = str(z)
    return 'http://webrd0'+port+'.is.autonavi.com/appmaptile?x='+x+'&y='+y+'&z='+z+'&lang=zh_cn&size=1&scl=1&style=8'
def createLocalFile(x,y,z,cacheDir):
    #'''创建缓存本地路径'''

    #拼装本地路径
    return cacheDir + os.sep + z + os.sep + y + os.sep + x + '.png'

def loadTask(task,threadNum):
    global tasksize
    #if tasksize:
    tasksize += tempTask.qsize()


    print (u'待下载Tile总计:%s,下载线程数:%s'%(tasksize,threadNum))
    print (u'开始下载')
    print (u'下载中...')


    for i in range(threadNum):
        Download().start() #生成多个线程并启动
    if out_type == 'sqlite':
        Save_db().start()


def geo_lng_lat(input_list):
    output_list = []
    tmp = []
    n = 0
    while True:
        if input_list == []:
            break
        for index, i in enumerate(input_list):
            if type(i)== list:
                input_list = i + input_list[index+1:]
                break
            else:
                output_list.append(i)
                n += 1
                input_list.pop(index)
                break

        if n % 2 == 0 and n > 0 and output_list != []:
            tmp.append(output_list)
            output_list = []

    lng_tmp = []
    lat_tmp = []
    for rows in tmp:
        lng_tmp.append(rows[0])
        lat_tmp.append(rows[1])
    re = '%f %f %f %f' %(min(lng_tmp),min(lat_tmp),max(lng_tmp),max(lat_tmp))
    return re

class Download(threading.Thread):
    progress = 0
    failureCount = 0

    def run(self):        

        sreq = requests.Session()#利用会话减少网络连接开销
        a = requests.adapters.HTTPAdapter(pool_connections = 1000, pool_maxsize = 1000,max_retries=3)
        sreq.mount('http://', a)
        while 1:
            try:
                item = tempTask.get(timeout=5)
            except Exception  as e:
                break
            valueAry = item.split(",")
            lv = valueAry[0]
            row = valueAry[1]
            col = valueAry[2]
            out_type = valueAry[3]
            try_nums = int(valueAry[4])
            remoteFile = createRemoteUrl(col,row,lv)
            print (remoteFile)
            localFile = createLocalFile(col,row,lv,cacheDir)
            try:
                if out_type == 'file':
                    r = sreq.get(remoteFile)
                    with open(localFile,'wb') as f:
                        f.write(r.content)
                        f.close()
                elif  out_type == 'sqlite':
                    binary_data = sreq.get(remoteFile).content
                    savedb.put('%s,%s,%s,%s' %(lv,row,col,str(base64.b64encode(binary_data),'utf8')))
                else:
                    print ('文件保存方式错误:%s' %out_type)
            except Exception  as e:
                print(e)
                if try_nums > try_num: #超过最大错误次数写入日志文件
                    logStr = '%s,%s,%s,0'%(lv,row,col)
                    f = open('./error.log','a')#在日志文件中打印失败记录
                    f.write(item+'\n')
                    f.close()
                else:
                    try_nums += 1
                    item = '%s,%s,%s,%s,%d' %(lv,row,col,out_type,try_nums)
                    tempTask.put(item) # 没超过的重新推入队列

                Download.failureCount += 1
                print (u'下载失败数量：%s'%Download.failureCount )#记录下载失败数量

            Download.progress = Download.progress + 1 #记录下载总数量，包括失败数量
            if Download.progress%100 == 0:
                print (u'下载进度为:%s/%s'%(Download.progress,tasksize))
        print('Download线程退出')


class Save_db(threading.Thread):

    def init_db(self,db_file):
        '''
        @param path:新建数据库路径
        @param name: 数据库名称 
        @return: 返回新建的数据库
        '''
        conn = sqlite3.connect(db_file + ".sqlite")
        c = conn.cursor()
        c.execute('''select count(*) from sqlite_master where type='table' and tbl_name='tiles' ''')
        if c.fetchall()[0][0] == 0:
            c.execute('''CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);''')
            c.execute('''CREATE UNIQUE INDEX "main"."tile_index"
    ON "tiles" ("zoom_level" ASC, "tile_column" ASC, "tile_row" ASC);''')
        conn.commit()
        c.close()
        return conn

    def run(self):        
        conn = self.init_db(db_file)
        c = conn.cursor()
        c.execute('PRAGMA synchronous = OFF')
        c.execute('PRAGMA journal_mode = OFF;')
        n = 0
        records_data = []
        while 1:
            try:
                item = savedb.get(timeout=5)
            except Exception  as e:
                if tempTask.qsize() == 0:
                    break
                else:
                    continue

            valueAry = item.split(",")
            z = valueAry[0]
            y = valueAry[1]
            x = valueAry[2]
            data = base64.b64decode(valueAry[3])
            records_data.append((z,x,y,sqlite3.Binary(data)))
            sql = "insert into tiles (zoom_level,tile_column,tile_row,tile_data) values (?,?,?,?)"
            #print(sql)
            #print(len(records_data))
            if len(records_data) == 1000:
                try:
                    c.executemany(sql, records_data,)
                    conn.commit()
                    records_data = []
                except Exception  as e:
                    print(e)
                    print(valueAry)
                    print(sql)

        c.executemany(sql, records_data,)
        conn.commit()
        c.execute('''select count(*) from tiles ''')
        print (c.fetchall()[0][0])
        c.close() 
        print('Save_db线程退出')


if __name__ == '__main__':
    global tempTask
    global tasksize
    global savedb
    global db_file
    global try_num1
    global out_type
    tasksize = 0
    tempTask = queue.Queue()    #事先把队列创建好之后再创建并启动线程，保证qsize正常
    savedb = queue.Queue()
    tasks = []
    extent = '-179.999999 -84.999999 179.999999 84.999999' #全球瓦片下载0-7
    minLv = 0 
    maxLv = 7  #最大区域范围，切记每个等级不要有重复，不然后续等级下载会失败
    ext = [float(i) for i in extent.split(' ')]
    extAry = [ext[0],ext[3],ext[2],ext[1]]          #区域范围
    lvRange = range(int(minLv),int(maxLv) + 1)      #等级范围
    tasks.append([extAry,lvRange])

    extent = '73.228179 3.832286 136.509429 54.349088'  #中国瓦片下载8-12
    minLv = 8
    maxLv = 12
    ext = [float(i) for i in extent.split(' ')]
    extAry = [ext[0],ext[3],ext[2],ext[1]]          #区域范围
    lvRange = range(int(minLv),int(maxLv) + 1)      #等级范围
    tasks.append([extAry,lvRange])
    
    #geojson下载地址：http://datav.aliyun.com/tools/atlas/#&lat=33.89527433526542&lng=106.720060693723&zoom=4.5 #下载时取消勾选包含子区域
    with  open('./geojson/jining.geojson',encoding='UTF-8') as f:
        lines = f.readlines()
        f.close()
    geo = json.loads(''.join(lines).replace('\n',''))['features'][0]['geometry']['coordinates']
    extent = geo_lng_lat(geo)
    print (extent)
    minLv = 13
    maxLv = 18
    ext = [float(i) for i in extent.split(' ')]
    extAry = [ext[0],ext[3],ext[2],ext[1]]          #区域范围
    lvRange = range(int(minLv),int(maxLv) + 1)      #等级范围
    tasks.append([extAry,lvRange])



    out_type = 'file' #输出格式 file/sqlite
    cacheDir = './out' #下载目录
    threadNum = 32 #下载线程
    db_file = './test' #sqllite数据库文件前缀
    try_num = 99 #下载失败重试次数
    if out_type == 'sqlite' and os.path.isfile('./'+ db_file + ".sqlite"):
        print('数据库文件已存在，请移除后重试！！')
        sys.exit()
    if os.path.isfile('./error.log'):
        with  open('./error.log','r+',encoding='UTF-8') as f:
            lines = f.readlines()

            if len(lines) > 0:
                key = input('发现任务失败文件%d行，是否继续下载Y/N:' %len(lines))
                if key.lower() == 'y':
                    f.seek(0)
                    f.truncate() #清空错误文件
                    for line in lines:
                        tempTask.put(line.replace('\n',''))
                    loadTask(tempTask,int(threadNum))
                else:
                    print('请清空任务失败文件后重新执行下载！！')
                    sys.exit()
                
                f.close()
            else:
                for tt in tasks:
                    createCacheStruc(tt[0],tt[1],cacheDir,out_type)       #创建缓存目录结构
                loadTask(tempTask,int(threadNum))                    #下载
 
    else:
        for tt in tasks:
            createCacheStruc(tt[0],tt[1],cacheDir,out_type)       #创建缓存目录结构
        print(tempTask.qsize())
        loadTask(tempTask,int(threadNum))                    #下载



     